#!/usr/bin/python
# -*- coding: UTF-8 -*-

################################################################################
#
# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved
#
################################################################################
"""
Interaction between file and mysql data

Authors: xiaojianghui
Date:    10/22/2020 11:01 AM
"""

import requests
from lxml import etree
from datetime import datetime


def crawling(url):
    """
    Grab cve specific information
    :param url: string
    :return xpth_list: list
    """
    xpth_list = []
    if url is None or url == "" or url.find("http") == -1:
        print("crawling, url:", url)
        return
    try:
        content = requests.get(url).content
    except requests.exceptions.ConnectionError:
        print('ConnectionError')
        return []
    except requests.exceptions.ChunkedEncodingError:
        print('ChunkedEncodingError')
        return []
    if content and len(content) > 1:
        html = etree.HTML(content)
        try:
            # if html.xpath(
            #         '/html/body/div[2]/div[2]/div/table/tr/td/div/div[1]/div[3]/div[2]/div[1]'
            #         '/div[2]/span/span/a/text()') == [
            #     "N/A"] or \
            #         html.xpath(
            #             '/html/body/div[2]/div[2]/div/table/tr/td/div/div[1]/div[2]/div[2]/div[1]'
            #             '/div[2]/span/span/a/text()') == [
            #     "N/A"] or \
            #         html.xpath(
            #             '/html/body/div[2]/div[2]/div[2]/table/tr/td/div/div[1]/div[4]/div[2]/div[1]/div[2]'
            #             '/span/span/a/text()') == ['N/A']:
            #     if html.xpath(
            #             "/html/body/div[2]/div[2]/div/table/tr/td/div/div[1]/div[2]/div[3]/div[1]/div[2]"
            #             "/span/span/a/text()") == [
            #         "N/A"] or \
            #             html.xpath(
            #                 "/html/body/div[2]/div[2]/div/table/tbody/tr/td/div/div[1]/div[2]/div[2]/div[1]"
            #                 "/div[2]/span/span/a/text()") == ["N/A"]:
            #         nvd_score = cve_level = cve_desc = repair_time = vector_value = attack_vector = \
            #             access_vector = attack_complexity = access_complexity = \
            #             privilege_required = user_interaction = scope = confidentiality = \
            #             integrity = availability = authentication = None
            #         print("No data on this vulnerability link, ", url)
            #         score_type = ""
            #         cve_desc = str(html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()')[0])
            #         if cve_desc:
            #             score_type = "v3.0"
            #     else:
            #         score_type = "v2.0"
            #         element = html.xpath('//*[@id="nistV2MetricHidden"]/@value')
            #         cve_desc = str(html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()')[0])
            #         repair_time = str(
            #             html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[2]/div/span[1]/text()')[0])
            #         if repair_time is not None:
            #             repair_time = datetime.strptime(repair_time, '%m/%d/%Y')
            #         html1 = etree.HTML(element[0])
            #         cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv2-base-score-severity"]/text()')
            #                         [0].strip()).capitalize()
            #         nvd_score = str(html1.xpath('//*[@data-testid="vuln-cvssv2-base-score"]/text()')[0].strip())
            #         vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv2-vector"]/text()')[0]). \
            #             replace("(", "").replace(")", "").strip()
            #         access_vector = str(html1.xpath('//*[@data-testid="vuln-cvssv2-av"]/text()')[0].strip())
            #         access_complexity = str(html1.xpath('//*[@data-testid="vuln-cvssv2-ac"]/text()')[0].strip())
            #         authentication = str(html1.xpath('//*[@data-testid="vuln-cvssv2-au"]/text()')[0].strip())
            #         confidentiality = str(html1.xpath('//*[@data-testid="vuln-cvssv3-c"]/text()')[0].strip())
            #         integrity = str(html1.xpath('//*[@data-testid="vuln-cvssv2-i"]/text()')[0].strip())
            #         availability = str(html1.xpath('//*[@data-testid="vuln-cvssv2-a"]/text()')[0].strip())
            #         attack_vector = attack_complexity = privilege_required = user_interaction = scope = None
            # elif html.xpath(
            #         '/html/body/div[2]/div[2]/div/table/tr/td/div/div[1]/div[3]/div[2]/div[1]/div[2]'
            #         '/span/span/a/text()') == [] and \
            #         html.xpath(
            #             '/html/body/div[2]/div[2]/div/table/tr/td/div/div[1]/div[2]/div[2]/div[1]'
            #             '/div[2]/span/span/a/text()') == []:
            #     nvd_score = cve_level = cve_desc = repair_time = vector_value = attack_vector = \
            #         access_vector = attack_complexity = access_complexity = \
            #         privilege_required = user_interaction = scope = confidentiality = integrity = \
            #         availability = authentication = None
            #     score_type = "v3.0"
            #     print("This vulnerability link not found, ", url)
            # else:
            #     score_type = "v3.0"
            #     cve_desc = str(html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()')[0])
            #     repair_time = html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[2]/div/span[1]/text()')[0]
            #     if repair_time is not None:
            #         repair_time = datetime.strptime(repair_time, '%m/%d/%Y')
            #     if html.xpath('//*[@id="nistV3MetricHidden"]/@value'):
            #         element = html.xpath('//*[@id="nistV3MetricHidden"]/@value')
            #     else:
            #         element = html.xpath('//*[@id="cnaV3MetricHidden"]/@value')
            #     html1 = etree.HTML(element[0])
            #     cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score-severity"]/text()')[0]
            #                     .strip()).capitalize()
            #     nvd_score = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score"]/text()')[0].strip())
            #     vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv3-vector"]/text()')[0]).replace("(", ''). \
            #         replace(')', '').strip()
            #     attack_vector = str(html1.xpath('//*[@data-testid="vuln-cvssv3-av"]/text()')[0].strip())
            #     attack_complexity = str(html1.xpath('//*[@data-testid="vuln-cvssv3-ac"]/text()')[0].strip())
            #     privilege_required = str(html1.xpath('//*[@data-testid="vuln-cvssv3-pr"]/text()')[0].strip())
            #     user_interaction = str(html1.xpath('//*[@data-testid="vuln-cvssv3-ui"]/text()')[0].strip())
            #     scope = str(html1.xpath('//*[@data-testid="vuln-cvssv3-s"]/text()')[0].strip())
            #     confidentiality = str(html1.xpath('//*[@data-testid="vuln-cvssv3-c"]/text()')[0].strip())
            #     integrity = str(html1.xpath('//*[@data-testid="vuln-cvssv3-i"]/text()')[0].strip())
            #     availability = str(html1.xpath('//*[@data-testid="vuln-cvssv3-a"]/text()')[0].strip())
            #     access_vector = access_complexity = authentication = None
            nvd_score = cve_level = cve_desc = repair_time = vector_value = attack_vector = \
                access_vector = attack_complexity = access_complexity = \
                privilege_required = user_interaction = scope = confidentiality = integrity = \
                availability = authentication = None
            cve_descx = html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()')
            if cve_descx is not None and len(cve_descx) > 0:
                cve_desc = str(cve_descx[0])
            # repair_timex = html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[2]/div/span[1]/text()')
            repair_timex = html.xpath('//*[@data-testid="vuln-published-on"]/text()')
            if repair_timex is not None and len(repair_timex) > 0:
                repair_time = str(repair_timex[0])
            if repair_time is not None and repair_time != "":
                repair_time = str(datetime.strptime(repair_time, '%m/%d/%Y'))
            score_type = "v3.0"
            if html.xpath('//*[@id="nistV3MetricHidden"]/@value'):
                element = html.xpath('//*[@id="nistV3MetricHidden"]/@value')
            else:
                element = html.xpath('//*[@id="cnaV3MetricHidden"]/@value')
            if element and len(element) > 0:
                html1 = etree.HTML(element[0])
                if html1 is not None:
                    cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score-severity"]/text()')[0]
                                    .strip()).capitalize()
                    nvd_score = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score"]/text()')[0].strip())
                    vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv3-vector"]/text()')[0]).replace("(",
                                                                                                                ''). \
                        replace(')', '').strip()
                    attack_vector = str(html1.xpath('//*[@data-testid="vuln-cvssv3-av"]/text()')[0].strip())
                    attack_complexity = str(html1.xpath('//*[@data-testid="vuln-cvssv3-ac"]/text()')[0].strip())
                    privilege_required = str(html1.xpath('//*[@data-testid="vuln-cvssv3-pr"]/text()')[0].strip())
                    user_interaction = str(html1.xpath('//*[@data-testid="vuln-cvssv3-ui"]/text()')[0].strip())
                    scope = str(html1.xpath('//*[@data-testid="vuln-cvssv3-s"]/text()')[0].strip())
                    confidentiality = str(html1.xpath('//*[@data-testid="vuln-cvssv3-c"]/text()')[0].strip())
                    integrity = str(html1.xpath('//*[@data-testid="vuln-cvssv3-i"]/text()')[0].strip())
                    availability = str(html1.xpath('//*[@data-testid="vuln-cvssv3-a"]/text()')[0].strip())
                    access_vector = access_complexity = authentication = None
            else:
                element = html.xpath('//*[@id="nistV2MetricHidden"]/@value')
                if element and len(element) > 0:
                    html1 = etree.HTML(element[0])
                    if html1 is not None:
                        score_type = "v2.0"
                        cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv2-base-score-severity"]/text()')
                                        [0].strip()).capitalize()
                        nvd_score = str(html1.xpath('//*[@data-testid="vuln-cvssv2-base-score"]/text()')[0].strip())
                        vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv2-vector"]/text()')[0]). \
                            replace("(", "").replace(")", "").strip()
                        access_vector = str(html1.xpath('//*[@data-testid="vuln-cvssv2-av"]/text()')[0].strip())
                        access_complexity = str(html1.xpath('//*[@data-testid="vuln-cvssv2-ac"]/text()')[0].strip())
                        authentication = str(html1.xpath('//*[@data-testid="vuln-cvssv2-au"]/text()')[0].strip())
                        confidentiality = str(html1.xpath('//*[@data-testid="vuln-cvssv3-c"]/text()')[0].strip())
                        integrity = str(html1.xpath('//*[@data-testid="vuln-cvssv2-i"]/text()')[0].strip())
                        availability = str(html1.xpath('//*[@data-testid="vuln-cvssv2-a"]/text()')[0].strip())
                        attack_vector = attack_complexity = privilege_required = user_interaction = scope = None
            if cve_desc == 'N/A':
                cve_desc = None
            if repair_time == 'N/A':
                repair_time = None
            if nvd_score is None or nvd_score == "" or nvd_score == 'N/A':
                nvd_score = None
            print("nvd_score:", nvd_score, "\n", "cve_level:", cve_level, "\n",
                  "repair_time:", repair_time, "\n", "score_type:", score_type, "\n",
                  "vector_value, attack_vector, access_vector,attack_complexity, \n"
                  "access_complexity, privilege_required, user_interaction, scope,\n"
                  "confidentiality, integrity, availability, authentication:\n",
                  vector_value, attack_vector, access_vector,
                  attack_complexity, access_complexity, privilege_required, user_interaction, scope,
                  confidentiality, integrity, availability, authentication, "\n", "cve_desc:", cve_desc)
            xpth_list = [nvd_score, cve_level, cve_desc, repair_time, vector_value, attack_vector, access_vector,
                         attack_complexity, access_complexity, privilege_required, user_interaction, scope,
                         confidentiality, integrity, availability, authentication, score_type]
        except IndexError as e:
            print("Subscript out of bounds", e)
        except UnboundLocalError as e:
            print("Tag not found", e)
    return xpth_list
